{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "lesbian-springer",
   "metadata": {},
   "source": [
    "## Metric Examples\n",
    "\n",
    "This notebook tests the exposed Prometheus metrics of model and pipeline servers.\n",
    "\n",
    "Requires: `prometheus_client` and `requests` libraries.\n",
    "See docs for full set of metrics available."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "2abedfc8",
   "metadata": {},
   "outputs": [],
   "source": [
    "mlserver_metrics_host=\"0.0.0.0:9006\"\n",
    "triton_metrics_host=\"0.0.0.0:9007\"\n",
    "pipeline_metrics_host=\"0.0.0.0:9009\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "f93e80df",
   "metadata": {},
   "outputs": [],
   "source": [
    "from prometheus_client.parser import text_string_to_metric_families\n",
    "import requests\n",
    "\n",
    "def scrape_metrics(host):\n",
    "    data = requests.get(f\"http://{host}/metrics\").text\n",
    "    return {\n",
    "        family.name: family for family in text_string_to_metric_families(data)\n",
    "    }\n",
    "\n",
    "def print_sample(family, label, value):\n",
    "    for sample in family.samples:\n",
    "        if sample.labels[label] == value:\n",
    "            print(sample)\n",
    "\n",
    "def get_model_infer_count(host, model_name):\n",
    "    metrics = scrape_metrics(host)\n",
    "    family = metrics[\"seldon_model_infer\"]\n",
    "    print_sample(family, \"model\", model_name)\n",
    "    \n",
    "def get_pipeline_infer_count(host, pipeline_name):\n",
    "    metrics = scrape_metrics(host)\n",
    "    family = metrics[\"seldon_pipeline_infer\"]\n",
    "    print_sample(family, \"pipeline\", pipeline_name)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d2bad42e",
   "metadata": {},
   "source": [
    "### MLServer Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "checked-cream",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{}\n",
      "{}\n"
     ]
    }
   ],
   "source": [
    "!seldon model load -f ./models/sklearn-iris-gs.yaml\n",
    "!seldon model status iris -w ModelAvailable | jq -M ."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "upper-cholesterol",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Success: map[:iris_1::50]\r\n"
     ]
    }
   ],
   "source": [
    "!seldon model infer iris -i 50 \\\n",
    "  '{\"inputs\": [{\"name\": \"predict\", \"shape\": [1, 4], \"datatype\": \"FP32\", \"data\": [[1, 2, 3, 4]]}]}' "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "right-talent",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Success: map[:iris_1::100]\r\n"
     ]
    }
   ],
   "source": [
    "!seldon model infer iris --inference-mode grpc -i 100 \\\n",
    "   '{\"model_name\":\"iris\",\"inputs\":[{\"name\":\"input\",\"contents\":{\"fp32_contents\":[1,2,3,4]},\"datatype\":\"FP32\",\"shape\":[1,4]}]}' "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "77205b71",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Sample(name='seldon_model_infer_total', labels={'code': '200', 'method_type': 'rest', 'model': 'iris', 'model_internal': 'iris_1', 'server': 'mlserver', 'server_replica': '0'}, value=50.0, timestamp=None, exemplar=None)\n",
      "Sample(name='seldon_model_infer_total', labels={'code': 'OK', 'method_type': 'grpc', 'model': 'iris', 'model_internal': 'iris_1', 'server': 'mlserver', 'server_replica': '0'}, value=100.0, timestamp=None, exemplar=None)\n"
     ]
    }
   ],
   "source": [
    "get_model_infer_count(mlserver_metrics_host,\"iris\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "ceramic-illness",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{}\r\n"
     ]
    }
   ],
   "source": [
    "!seldon model unload iris"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "30431134",
   "metadata": {},
   "source": [
    "### Triton Model"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "275a8445",
   "metadata": {},
   "source": [
    "Load the model."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "726f2dd0",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{}\n",
      "{}\n"
     ]
    }
   ],
   "source": [
    "!seldon model load -f ./models/tfsimple1.yaml\n",
    "!seldon model status tfsimple1 -w ModelAvailable | jq -M ."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "7abd240a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Success: map[:tfsimple1_1::50]\r\n"
     ]
    }
   ],
   "source": [
    "!seldon model infer tfsimple1 -i 50\\\n",
    "    '{\"inputs\":[{\"name\":\"INPUT0\",\"data\":[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16],\"datatype\":\"INT32\",\"shape\":[1,16]},{\"name\":\"INPUT1\",\"data\":[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16],\"datatype\":\"INT32\",\"shape\":[1,16]}]}'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "f86c494a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Success: map[:tfsimple1_1::100]\r\n"
     ]
    }
   ],
   "source": [
    "!seldon model infer tfsimple1 --inference-mode grpc -i 100 \\\n",
    "    '{\"model_name\":\"tfsimple1\",\"inputs\":[{\"name\":\"INPUT0\",\"contents\":{\"int_contents\":[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16]},\"datatype\":\"INT32\",\"shape\":[1,16]},{\"name\":\"INPUT1\",\"contents\":{\"int_contents\":[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16]},\"datatype\":\"INT32\",\"shape\":[1,16]}]}'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "2f0f940f",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Sample(name='seldon_model_infer_total', labels={'code': '200', 'method_type': 'rest', 'model': 'tfsimple1', 'model_internal': 'tfsimple1_1', 'server': 'triton', 'server_replica': '0'}, value=50.0, timestamp=None, exemplar=None)\n",
      "Sample(name='seldon_model_infer_total', labels={'code': 'OK', 'method_type': 'grpc', 'model': 'tfsimple1', 'model_internal': 'tfsimple1_1', 'server': 'triton', 'server_replica': '0'}, value=100.0, timestamp=None, exemplar=None)\n"
     ]
    }
   ],
   "source": [
    "get_model_infer_count(triton_metrics_host,\"tfsimple1\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "f7405940",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{}\r\n"
     ]
    }
   ],
   "source": [
    "!seldon model unload tfsimple1"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2d07a12e",
   "metadata": {},
   "source": [
    "### Pipeline\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "d540b7f0",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{}\n",
      "{}\n",
      "{}\n",
      "{}\n",
      "{}\n",
      "{\"pipelineName\":\"tfsimples\", \"versions\":[{\"pipeline\":{\"name\":\"tfsimples\", \"uid\":\"cdqji39qa12c739ab3o0\", \"version\":2, \"steps\":[{\"name\":\"tfsimple1\"}, {\"name\":\"tfsimple2\", \"inputs\":[\"tfsimple1.outputs\"], \"tensorMap\":{\"tfsimple1.outputs.OUTPUT0\":\"INPUT0\", \"tfsimple1.outputs.OUTPUT1\":\"INPUT1\"}}], \"output\":{\"steps\":[\"tfsimple2.outputs\"]}, \"kubernetesMeta\":{}}, \"state\":{\"pipelineVersion\":2, \"status\":\"PipelineReady\", \"reason\":\"created pipeline\", \"lastChangeTimestamp\":\"2022-11-16T19:25:01.255955114Z\"}}]}\n"
     ]
    }
   ],
   "source": [
    "!seldon model load -f ./models/tfsimple1.yaml \n",
    "!seldon model load -f ./models/tfsimple2.yaml\n",
    "!seldon model status tfsimple1 -w ModelAvailable | jq -M .\n",
    "!seldon model status tfsimple2 -w ModelAvailable | jq -M .\n",
    "!seldon pipeline load -f ./pipelines/tfsimples.yaml\n",
    "!seldon pipeline status tfsimples -w PipelineReady"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "1c78ca44",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Success: map[:tfsimple1_1::50 :tfsimple2_1::50 :tfsimples.pipeline::50]\r\n"
     ]
    }
   ],
   "source": [
    "!seldon pipeline infer tfsimples -i 50 \\\n",
    "    '{\"inputs\":[{\"name\":\"INPUT0\",\"data\":[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16],\"datatype\":\"INT32\",\"shape\":[1,16]},{\"name\":\"INPUT1\",\"data\":[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16],\"datatype\":\"INT32\",\"shape\":[1,16]}]}'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "7f48b7c2",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Sample(name='seldon_pipeline_infer_total', labels={'code': '200', 'method_type': 'rest', 'pipeline': 'tfsimples', 'server': 'pipeline-gateway'}, value=50.0, timestamp=None, exemplar=None)\n"
     ]
    }
   ],
   "source": [
    "get_pipeline_infer_count(pipeline_metrics_host,\"tfsimples\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "2a35f0ee",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{}\n",
      "{}\n",
      "{}\n"
     ]
    }
   ],
   "source": [
    "!seldon model unload tfsimple1\n",
    "!seldon model unload tfsimple2\n",
    "!seldon pipeline unload tfsimples"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "305f4ec9",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
